package com.uxbee.service;

import java.util.ArrayList;
import java.util.Comparator;
import java.util.Hashtable;
import java.util.List;
import java.util.Vector;
import java.util.concurrent.PriorityBlockingQueue;

import org.htmlparser.Node;
import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.filters.AndFilter;
import org.htmlparser.filters.HasAttributeFilter;
import org.htmlparser.filters.NodeClassFilter;
import org.htmlparser.filters.OrFilter;
import org.htmlparser.filters.TagNameFilter;
import org.htmlparser.nodes.TagNode;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.util.NodeIterator;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;

import com.uxbee.pojo.HtmlData;

// 获取WEB信息
public class GertWebInfo extends Thread {

	// 同步对象
	private static Byte b = new Byte("0");

	// 设置宽度
	private static int width = Integer.MAX_VALUE;
	
	// 设置深度
	private static int height = Integer.MAX_VALUE;
	
	// 初始化队列
	private static List<HtmlData> initList = new Vector<HtmlData>();

	// 完成的URL地址(同步哈希表)
	private static Hashtable<String, String> finishWebUrl = new Hashtable<String, String>();

	// 等待处理的URL地址(同步优先级队列)
	private static PriorityBlockingQueue<HtmlData> queue = new PriorityBlockingQueue<HtmlData>(
			100000, new Comparator<HtmlData>() {
				public int compare(HtmlData o1, HtmlData o2) {
					if (o1 == null && o2 == null) {
						return 0;
					} else if (o1 == null) {
						return -1;
					} else if (o2 == null) {
						return 1;
					}
					return o1.getLevel() - o2.getLevel();
				}
			});

	// 设置初始化数据
	public static void setInitData(HtmlData htmlData) {
		if (htmlData != null) {
			synchronized (b) {
				try {
					initList.add(htmlData);
				} catch (Exception e) {

				}
			}
		}
	}

	// 设置初始化数据
	public static void setInitData(List<HtmlData> list) {
		if (list != null && list.size() > 0) {
			synchronized (b) {
				try {
					for (int i = 0; i < list.size(); i++) {
						initList.add(list.get(i));
					}
				} catch (Exception e) {

				}
			}
		}
	}

	// 获取信息
	public void run() {
		while (true) {
			// 队列为空
			if (queue.size() == 0 && initList.size() == 0) {
				try {
					sleep(5000L);
					//System.out.println("队列为空系统休眠!");
				} catch (InterruptedException e) {
					e.printStackTrace();
				}
			}
			// 队列不为空进行页面抓取操作
			else {
				HtmlData htmlData = null;
				synchronized (b) {
					try {
						// 初始化数据
						if (initList != null){
							for (int i = 0; i < initList.size(); i++) {
								queue.put(initList.get(i));
							}
							initList.clear();
						}
						htmlData = GertWebInfo.queue.poll();
						// 设置处理完成标志
						finishWebUrl.put(htmlData.getUrl(), "");
					} catch (Exception e) {

					}
				}

				// 获取单个页面链接
				List<HtmlData> htmlDataList = null;
				try {
					Parser parser = new Parser(htmlData.getUrl());
					String encoding = getCode(parser);
					parser.reset();
					if (encoding != null && encoding.length() > 0) {
						parser.setEncoding(encoding);
					}

					// 获取页面内容
					StringBuffer webHtml = new StringBuffer();
					NodeIterator nodeIterator = parser.elements();
					while (nodeIterator.hasMoreNodes()) {
						Node tempNode = nodeIterator.nextNode();
						webHtml.append(tempNode.toHtml());
					}
					parser.reset();
					// 保存页面内容到数据库
					String html = webHtml.toString();
					// 加载内容判断模块
					CheckData checkData = (CheckData) Class.forName("com.uxbee.service.ch").newInstance();
					if (checkData.checkData(html)){
						// 保存处理类
						System.out.println(htmlData.getUrl() + "   --->   保存到数据库");
					}else{
						System.out.println(htmlData.getUrl() + "   --->   放弃数据");
					}

					// 获取标签过滤条件
					OrFilter linkFilter = getOrFilter();
					// 得到所有经过过滤的标签
					NodeList list = parser.extractAllNodesThatMatch(linkFilter);
					if (list == null || list.size() == 0) {
						Parser rep = new Parser(htmlData.getUrl());
						if (encoding != null && encoding.length() > 0) {
							rep.setEncoding(encoding);
						}
						NodeFilter ret = new TagNameFilter("body");
						list = rep.extractAllNodesThatMatch(ret);
					}
					LinkFilter filter = new LinkFilter() {

						public boolean accept(String mainUrl, String url) {
							int index = url.indexOf(mainUrl);
							if (index == -1) {
								return false;
							} else {
								return true;
							}

						}
					};
					// 获取不重复连接
					htmlDataList = getUrl(htmlData, filter, list, width);
					// 获取到的连接增加到队列
					if (htmlDataList != null && htmlDataList.size() > 0) {
						synchronized (b) {
							try {
								for (int i = 0; i < htmlDataList.size(); i++) {
									queue.put(htmlDataList.get(i));
								}
							} catch (Exception e) {

							}
						}
					}
				} catch (Exception e) {
					e.printStackTrace();
				}
			}
		}
	}

	// 设置过滤标签
	private OrFilter getOrFilter() {
		// 过滤 <frame >标签的 filter，用来提取 frame 标签里的 src 属性所表示的链接
		NodeFilter frameFilter = new NodeFilter() {

			private static final long serialVersionUID = -7074935221997870406L;

			public boolean accept(Node node) {
				if (node.getText().startsWith("frame src=")) {
					return true;
				} else {
					return false;
				}
			}
		};
		// OrFilter 来设置过滤 <a> 标签，和 <frame> 标签
		return new OrFilter(new NodeClassFilter(LinkTag.class), frameFilter);
	}

	// 获取页面编码兼容HTML5
	private String getCode(Parser parser) throws ParserException {

		Parser p = new Parser(parser.getURL());
		
		parser.reset();
		String encoding = parser.getEncoding();
		if (encoding != null){
			return encoding;
		}

		NodeFilter tagFilter = new TagNameFilter("meta");
/*		HasAttributeFilter haf1 = new HasAttributeFilter("charset");
		HasAttributeFilter haf2 = new HasAttributeFilter("http-equiv",
				"Content-Type");
		OrFilter orf = new OrFilter(haf1, haf2);
		AndFilter af = new AndFilter(tagFilter, orf);*/
		NodeList nodes = null;
		boolean isException = false;
		try {
			p.reset();
			nodes = p.extractAllNodesThatMatch(tagFilter);
			isException = false;
		} catch (Exception e) {
			isException = true;
		}
		if (isException) {
			try {
				p.reset();
				p.setEncoding("gbk");
				nodes = p.extractAllNodesThatMatch(tagFilter);
				isException = false;
			} catch (Exception e) {
				isException = true;
			}
		}
		if (isException) {
			try {
				p.reset();
				p.setEncoding("utf-8");
				nodes = p.extractAllNodesThatMatch(tagFilter);
				isException = false;
			} catch (Exception e) {
				isException = true;
			}
		}
		
		return encoding;
	}

	// 获取链接
	private List<HtmlData> getUrl(HtmlData htmlData, LinkFilter filter,
			NodeList list, int count) {
		List<HtmlData> htmlDataList = new ArrayList<HtmlData>();
		for (int i = 0; i < list.size(); i++) {
			if (htmlDataList.size() > count) {
				break;
			}
			Node tag = list.elementAt(i);
			if (tag instanceof LinkTag)// <a> 标签
			{
				LinkTag link = (LinkTag) tag;
				String linkUrl = link.getLink();// url
				if (linkUrl != null) {
					linkUrl = linkUrl.replaceAll("[#].*$", "");
				}
				if (filter.accept(htmlData.getMainUrl(), linkUrl)) {
					HtmlData tempHtmlData = new HtmlData();
					tempHtmlData.setUrl(linkUrl);
					tempHtmlData.setLevel(htmlData.getLevel() + 1);
					tempHtmlData.setMainUrl(htmlData.getMainUrl());
					boolean repair = false;
					// 链接重复的场合不保存链接
					for (int j = 0; j < htmlDataList.size(); j++) {
						HtmlData hd = htmlDataList.get(j);
						if (linkUrl.equals(hd.getUrl())) {
							repair = true;
							break;
						}
					}
					
					synchronized (b) {
						try {
							// 链接重复的场合不保存链接
							repair = repair || finishWebUrl.containsKey(linkUrl);
							
							repair = repair || queue.contains(tempHtmlData);
						} catch (Exception e) {

						}
					}
					
					if (!repair) {
						htmlDataList.add(tempHtmlData);
					}
				}
			} 
			// frame
			else {
				// 提取 frame 里 src 属性的链接如 <frame src="test.html"/>
				String frame = tag.getText();
				int start = frame.indexOf("src=");
				if (start == -1) {
					continue;
				}
				frame = frame.substring(start);
				int end = frame.indexOf(" ");
				if (end == -1) {
					end = frame.indexOf(">");
				}
				String frameUrl = frame.substring(5, end - 1);
				if (frameUrl != null) {
					frameUrl = frameUrl.replaceAll("[#].*$", "");
				}
				if (filter.accept(htmlData.getMainUrl(), frameUrl)) {
					HtmlData tempHtmlData = new HtmlData();
					tempHtmlData.setUrl(frameUrl);
					tempHtmlData.setLevel(htmlData.getLevel() + 1);
					tempHtmlData.setMainUrl(htmlData.getMainUrl());
					boolean repair = false;
					// 链接重复的场合不保存链接
					for (int j = 0; j < htmlDataList.size(); j++) {
						HtmlData hd = htmlDataList.get(j);
						if (frameUrl.equals(hd.getUrl())) {
							repair = true;
							break;
						}
					}
					
					synchronized (b) {
						try {
							// 链接重复的场合不保存链接
							repair = repair || finishWebUrl.containsKey(frameUrl);
							
							repair = repair || queue.contains(tempHtmlData);
						} catch (Exception e) {

						}
					}

					if (!repair) {
						htmlDataList.add(tempHtmlData);
					}
				}
			}
		}
		return htmlDataList;
	}
}
